# -*-coding:utf-8-*-
import sys
import requests
from lxml import etree
from props import properties
import re
from threads import book_task_fail_queue, save_queue
from crawler_base import crawler_base
from DAO.book import book
from DAO import proxies

props = properties.props
get_header = properties.get_header
get_proxies = proxies.get_proxy
reload(sys)
sys.setdefaultencoding('utf-8')


class book_list_crawler(crawler_base):
    bookPage_count = int(props.get('bookpage.count'))
    start_index = 0

    def __init__(self, suc=True, index=0):
        crawler_base.__init__(self, suc)
        self.start_index = index

    def get_booklist_bytags(self, current_tag, proxy):
        if props.get(current_tag + '_index') == '':
            props.put(current_tag + '_index', str(self.start_index))
        else:
            self.start_index = int(props.get(current_tag + '_index'))
        status = True
        while status:
            list_url = 'https://book.douban.com/tag/' + current_tag + '?start=' + str(self.start_index)
            status = self.get_booklist_perPage(list_url, proxy)
            props.put(current_tag + '_index', str(self.start_index))
        return self.success

    def get_booklist_perPage(self, page_url, proxy):
        tag = re.findall(r'tag/(.*?)\?', page_url)[0]
        try:
            tag_text = requests.get(page_url, headers=get_header(), proxies=proxy, timeout=5).content.decode("utf-8",
                                                                                                             'ignore')
            selector = etree.HTML(tag_text)
            bookList = selector.xpath('//*[@id="subject_list"]/ul/li')
        except Exception, e:
            print e.message
            try:
                book_task_fail_queue.put_nowait(tag)
            except Exception, e:
                print e.message
            self.success = False
            return False

        self.start_index += self.bookPage_count
        length = len(bookList)
        # crawling finished
        if length == 0:
            self.success = True
            return False
        for i in range(length):
            link = bookList[i].xpath('./div[2]/h2/a/@href')[0] + ''
            bookID = re.findall(r'subject/(.*?)/', link)[0]
            bookName = bookList[i].xpath('./div[2]/h2/a/text()')[0].replace('\n', '').replace(' ', '')
            newBook = book(bookID)
            newBook.book_name = bookName
            newBook.tag = tag
            print bookID + ':' + bookName

            save_queue.put(newBook)
        print 'list_end'
        self.success = True
        return True

    def work(self, param=None, proxy=None):
        self.success = self.get_booklist_bytags(param, proxy)
        return self.success
